package com.leaderment.timatt.service.impl;

import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;
import org.eclipse.jetty.util.StringUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.jsoup.nodes.Element;
import org.openqa.selenium.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.leaderment.timatt.SpringContextUtil;
import com.leaderment.timatt.mybatis.bo.CrawlerDomainProxyRedisMappingBO;
import com.leaderment.timatt.mybatis.entity.TrackingData;
import com.leaderment.timatt.mybatis.mapper.CrawlerDomainProxyRedisMappingMapper;
import com.leaderment.timatt.mybatis.mapper.CrawlerUrlMapper;
import com.leaderment.timatt.service.ICrawlerService;
import com.leaderment.timatt.webmagic.downloader.UpsSeleniumDownloader;
import com.leaderment.timatt.webmagic.extern.SeleniumAction;
import com.leaderment.timatt.webmagic.pageprocessor.UpsProcessor;
import com.leaderment.timatt.webmagic.pipeline.MyConsolePipeline;
import com.leaderment.timatt.webmagic.scheduler.MyQueueScheduler;

import us.codecraft.webmagic.Spider;


@Component
@SuppressWarnings({"rawtypes","unchecked"})
public class CrawlerServiceImpl implements ICrawlerService {

	@Autowired
    private RedisTemplate redisTemplate;
	@Autowired
	CrawlerUrlMapper crawlerUrlMapper;
	
	public static Spider oneSpider = null;
	
	Logger logger=Logger.getLogger(CrawlerServiceImpl.class);
	
	private String theUrl = "";
	
	public static LinkedBlockingQueue<Integer> threadBlockQueue = new LinkedBlockingQueue<>(1);//设置线程开启个数
	
	@Override
	@Scheduled(cron = "0 0 15 ? * * ")
	@SuppressWarnings("unused")
	public void startCrawler() {
		//由于UPS tracking 每条信息并非get请求,不能通过地址可以直接访问,需要post传入不同参数浏览不同页面.
		//所以重新定义了SelemiumAction 事件.
		
		while(true){
			try{
				// 线程每隔一段时间检查一下redis中的关键词
				Date nowDate=Calendar.getInstance().getTime();
				// 判断是否从redis中获取关键词
				this.firstUrlInit();
				if("".equals(theUrl))continue;
				List<String> trackingIds = crawlerUrlMapper.selectTrackingByStatus();
				this.getUpsTracking(trackingIds);
				
			}catch(Exception e){
				e.printStackTrace();
			}
		}
	}
	
		//初始化url
		public void firstUrlInit() {
			// 初始化map
			CrawlerDomainProxyRedisMappingMapper crawlerDomainProxyRedisMappingMapper=(CrawlerDomainProxyRedisMappingMapper)SpringContextUtil.getBean("crawlerDomainProxyRedisMappingMapper");
			List<CrawlerDomainProxyRedisMappingBO> list=crawlerDomainProxyRedisMappingMapper.selectDomainAndListName();
			for(int i=0;i<list.size();i++){
				if( "amzTrackingUPSList".equals(list.get(i).getCrawlerRedisListName())) {
					this.theUrl = list.get(i).getCrawlerDomain();
				}
			}
	    }
		/***
		 * review 爬虫方法
		 * 
		 * */
		@SuppressWarnings("unused")
		public void getUpsTracking(List<String> trackingIds) {
			while(true){
				try{
//					Thread.sleep(20000);
					//线程每隔一段时间检查一下redis中的关键词
					Date nowDate=null;
					//计数初始化为0
					int crawlerCount=0;
					//判断是否从redis中获取数据
					boolean getDataFlag=true;
//					List redisString=redis.opsForList().range(listName, 0, -1);
					//从redis中取出数据
					if(StringUtil.isBlank(this.theUrl))getDataFlag = false;
					while(getDataFlag){
							crawlerCount++;//关键词计数加1
							UpsSeleniumDownloader seleniumDownloader=new UpsSeleniumDownloader(new UpsAction(trackingIds));
							
							//开启爬虫线程
							oneSpider =Spider.create(new UpsProcessor(this.theUrl))
											.setDownloader(seleniumDownloader)
											.setScheduler(new MyQueueScheduler())//设置允许重复URL的队列，为了URL失败后重新放回
											.addPipeline(new MyConsolePipeline())
											;
							//爬虫线程不自动退出
							oneSpider.setExitWhenComplete(false);
							//队列空闲等待时间
							oneSpider.setEmptySleepTime(1000);
							//放入初始URL
							oneSpider.addUrl(this.theUrl)//初始URL
					         .thread(1)
					         .run();
							threadBlockQueue.put(1);
						
					}
				}catch(Exception e){
					e.printStackTrace();
					//停止所有爬虫线程
//					for (Entry<String, Spider> entry : spiderMap.entrySet()) {  
//						entry.getValue().stop();
//					} 
				}
				
			}
		}
		/**
		 * 滚动条下拉滚动到最底.
		 * */
		 private class UpsAction implements SeleniumAction{
			 
			 private List<String> theTrackingIds;
			 
			 public UpsAction(List<String> trackingIds) {
				 this.theTrackingIds = trackingIds;
			 }
			 
		        @SuppressWarnings("unchecked")
				@Override
		        public void execute(WebDriver driver) {
		        	WebElement inputElement =driver.findElement(By.id("stApp_trackingNumber"));

		        	//隐式渲染
					driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);

		        	//如果页面加载出来了才从redis里面取trackingId.
		        	if(null == this.theTrackingIds || this.theTrackingIds.size() <1) return;
		        	for(String oneId : theTrackingIds) {
		        		String theKey = oneId.replace("\"", "");
		        		inputElement.sendKeys(theKey);//输入一个trackingId,然后输入一个回车.
		        		inputElement.sendKeys(Keys.ENTER);
		        	}
		        	try{
			        	WebElement btnElement =driver.findElement(By.id("stApp_btnTrack"));
			        	if(null != btnElement )  btnElement.click();
			        	
			        	Thread.sleep(5000);
//			        	WebElement detailElement =driver.findElement(By.id("stApp_iconShipmentProgress"));
//			        	if(null != detailElement )  {
//			        		detailElement.click();
//			        		//进入此处,证明只找到了一条trackingId的记录信息.
//			        	}
			        		//进入此处,证明找到多条记录信息.需要循环获取每一条信息并获取下一页信息.
			        		//
//			        		WebElement webElement = driver.findElement(By.xpath("/html"));
//							String content = webElement.getAttribute("outerHTML");
//							Document parse = Jsoup.parse(content);
//			        		Elements cards =parse.getElementsByClass("ups-tracking_cards ups-card_list").select("li");
			        		WebElement cardOuterEle = driver.findElements(By.className("ups-tracking_cards")).get(0);
			        		List<WebElement> cards =  cardOuterEle.findElements(By.tagName("li"));
			        		
			        		if(null != cards && cards.size()>0) {
			        			for(int i = 0 ; i < cards.size() ; i++ ) {
									//隐式渲染
									driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
									if(i>0) {
			        					//如果不是首次加载,则需要重新加载.
			        					cardOuterEle = driver.findElements(By.className("ups-tracking_cards")).get(0);
			        					cards =  cardOuterEle.findElements(By.tagName("li"));
			        				} 
			        				WebElement card = cards.get(i);
			        				
			        				WebElement linkEl = card.findElements(By.className("ups-link")).get(0);
			        				if(null != linkEl) {
										try {
											linkEl.click();
										} catch (Exception e) {
											e.printStackTrace();
										}
										Thread.sleep(8000);
			        					WebElement detailElement =driver.findElement(By.id("stApp_iconShipmentProgress"));
			        					if(null != detailElement )detailElement.click();
			        					Thread.sleep(6000);
			        					
						        		WebElement webElement = driver.findElement(By.xpath("/html"));
										String content = webElement.getAttribute("outerHTML");
										Document parse = Jsoup.parse(content);
										String trackingId = "";	String trackingProcess = "";
										String firstVal=""; String secondVal=""; String thirdVal="";
										if (parse!=null) {
											//trackingId Element
											Element trackingIdElt = parse.getElementById("stApp_lblTrackingNumber");
											Elements progressElts =parse.getElementsByClass("ups-progress_current_row");
											if(null != trackingIdElt )trackingId = trackingIdElt.text().split("Number")[1].trim();
											if(progressElts!=null&&progressElts.size()>0){
												if(null != progressElts.select("td") && progressElts.select("td").size()>0) {
													for( int j = 0 ; j<progressElts.select("td").size() ; j++) {
														Element theEl = progressElts.select("td").get(j);
														if(j == 0) {
															firstVal = theEl.text();
														}else if(j == 1) {
															secondVal = theEl.text();
														}else if(j == 2) {
															thirdVal = theEl.text();
														}
													}
												}
											}
											if(!"".equals(firstVal) && !"".equals(secondVal) && !"".equals(thirdVal) )
												trackingProcess = secondVal + " " + firstVal + " " + thirdVal ;
											if(!"".equals(trackingProcess) && !"".equals(trackingId)) {
												TrackingData param = new TrackingData();
												
												param.setTrackingId(trackingId);
												param.setTrackingProcess(trackingProcess);
												
												//该bean用于持久化爬下的数据.
												CrawlerDomainProxyRedisMappingMapper crawlerDomainProxyRedisMappingMapper=(CrawlerDomainProxyRedisMappingMapper)SpringContextUtil.getBean("crawlerDomainProxyRedisMappingMapper");
												
												crawlerDomainProxyRedisMappingMapper.updateTrackingProcess(param);
											}
										}
			        					WebElement backElement =driver.findElement(By.id("stApp_btnBackToList"));
			        					if(null == backElement) {
			        						Thread.sleep(3000);
			        						backElement =driver.findElement(By.id("stApp_btnBackToList"));
			        						backElement.click();
			        					}else{
			        						/*//欲点击的未显示的控件在页面底部（顶部略）时，将页面滚动至底部
											//移动到元素element对象的“底端”与当前窗口的“底部”对齐
											((JavascriptExecutor) driver).executeScript("arguments[0].scrollIntoView(false);", backElement);*/
			        						backElement.click();
			        					}
			        					Thread.sleep(10000);
			        				}
			        			}
			        		}
			        		
			        	
			        	
		        	}catch(Exception e){
						e.printStackTrace();
						//停止所有爬虫线程
					}
		        }
		        // 写入方法
		    	private void forSendKeys(WebElement element,String content){
		    		for (int i = 0; i < content.length(); i++) {
		    			element.sendKeys(content.charAt(i)+"");
		    		}
		    	}
		    }
}
